import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import util.HttpClientUtil;

import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.PrintStream;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

public class MainApplication {

    public static final String PROVINCE_CODE = "provinceCode";
    public static final String PROVINCE_NAME = "provinceName";
    public static final String CITY_CODE = "cityCode";
    public static final String CITY_NAME = "cityName";
    public static final String COUNTY_CODE = "countyCode";
    public static final String COUNTY_NAME = "countyName";
    public static final String TOWN_CODE = "townCode";
    public static final String TOWN_NAME = "townName";
    public static final String VILLAGE_CODE = "villageCode";
    public static final String VILLAGE_NAME = "villageName";


    // 我是按照博客园的文章 用Java写的一个爬虫程序， 这里选了2020年的数据页
    public static final String url = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017/";
    private static final Map<String, String> headers = new HashMap<>();

    public static void main(String[] args) throws FileNotFoundException {

         String outputFile = "C:\\Users\\Administrator\\Desktop\\full.sql";
         System.setOut(new PrintStream(new FileOutputStream(outputFile)));

        // 31个省 + 自治区 + 直辖市
        final List<Map<String, String>> provinceList = new LinkedList<>();
        getProvinceList(url, provinceList);

        // 342个 市 + 直辖市区 + 自治区县
        final List<Map<String, String>> cityList = new LinkedList<>();
        provinceList.forEach(map -> {
            String url = map.get("link");
            if (null == url) return;
            getCityList(url, cityList, map);
        });

        // 3271个区县
        final List<Map<String, String>> countyList = new LinkedList<>();
        cityList.forEach(map -> {
            String url = map.get("link");
            if (null == url) return;
            try {
                Thread.sleep(500);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            getCountyList(url, countyList, map);
        });

        /**
         * 从这里开始不能直接获取了，可以尝试
         * 41536个 街道 + 乡镇 这里会被反爬拦截，不建议这样获取了，因为请求头没有任何处理
         */
//        final List<Map<String, String>> townList = new LinkedList<>();
//        countyList.forEach(map -> {
//            String url = map.get("link");
//            if (null == url) return;
//            try {
//                Thread.sleep(500);
//            } catch (InterruptedException e) {
//                e.printStackTrace();
//            }
//            getTownList(url, townList, map);
//        });

        // 最后 是 居委会 + 村委会 + 等等最小基层单位 数量未知， 这里也会被反爬拦截，也不建议这样获取了，因为请求头没有任何处理
//        final List<Map<String, String>> villageList = new LinkedList<>();
//        townList.forEach(map -> {
//            String url = map.get("link");
//            if (null == url) return;
//            try {
//                Thread.sleep(500);
//            } catch (InterruptedException e) {
//                e.printStackTrace();
//            }
//            getVillageList(url, villageList, map);
//        });
    }

    public static void getVillageList(String url, List<Map<String, String>> villageList, Map<String, String> town) {
        String htmlSourceCode = HttpClientUtil.executeGetRequest(url, headers);
        Document document = Jsoup.parse(htmlSourceCode);
        Elements towntr = document.getElementsByClass("villagetr");

        towntr.forEach(element -> {
            Elements tdTags = element.getElementsByTag("td");
            String villageCode = tdTags.get(0).text();
            String villageType = tdTags.get(1).text();
            String villageName = tdTags.get(2).text();

            Map<String, String> village = new HashMap<>();
            village.put(VILLAGE_NAME, villageName);
            village.put("villageType", villageType);
            village.put(VILLAGE_CODE, villageCode);
            // System.out.println(village);
            System.out.printf("(%s, '%s', '%s', NULL, '%s', NULL, NOW()),\n", villageCode, villageName, town.get(TOWN_CODE), villageType );
            villageList.add(village);
        });

    }

    public static void getTownList(String url, List<Map<String, String>> townList, Map<String, String> county) {
        String htmlSourceCode = HttpClientUtil.executeGetRequest(url, headers);
        Document document = Jsoup.parse(htmlSourceCode);
        Elements towntr = document.getElementsByClass("towntr");

        towntr.forEach(element -> {
            Elements tdTags = element.getElementsByTag("td");

            // System.out.println("--------------------------------------------------");
            // System.out.println(tdTags);

            String tdTag1 = tdTags.get(0).html();
            if (tdTag1.contains("href")) {
                Elements aTags = element.getElementsByTag("a");
                Element aTag1 = aTags.get(0);
                Element aTag2 = aTags.get(1);
                String href = aTag1.attr("href");
                String townCode = aTag1.text();
                String townName = aTag2.text();

                Map<String, String> town = new HashMap<>();
                town.put("townName", townName);
                town.put("townCode", townCode);
                //  MainApplication.url
                /**
                 *
                 * http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/13/29/130229401.html
                 * http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2020/13/02/29/130229104.html
                 */
                String code = href.split("/")[1];
                String code1 = code.substring(0,2) + "/";
                String code2 = code.substring(2,4) + "/";

                town.put("link",  MainApplication.url + code1 + code2 + href);
                townList.add(town);

                System.out.printf("(%s, '%s', '%s', NULL, NULL, '%s', NOW()),\n", townCode, townName, county.get(COUNTY_CODE), town.get("link"));
                // System.out.println(town);
            } else  {

                String townName = tdTags.get(1).text();

                Map<String, String> town = new HashMap<>();
                town.put("townName", townName);
                town.put("townCode", tdTag1);
                town.put("link", null);
                townList.add(town);

                System.out.printf("(%s, '%s', '%s', NULL, NULL, NULL, NOW()),\n", tdTag1, townName, county.get(COUNTY_CODE) );
            }
        });
    }

    public static void getCountyList(String url, List<Map<String, String>> countyList, Map<String, String> city) {
        String htmlSourceCode = HttpClientUtil.executeGetRequest(url, headers);
        Document document = Jsoup.parse(htmlSourceCode);
        Elements countytr = document.getElementsByClass("countytr");
        countytr.forEach(element -> {
            // 这一级开始存在叶子节点了
            Elements tdTags = element.getElementsByTag("td");
            // System.out.println("--------------------------------------------------");
            // System.out.println(tdTags);

            String tdTag1 = tdTags.get(0).html();
            if (tdTag1.contains("href")) {
                Elements aTags = element.getElementsByTag("a");
                Element aTag1 = aTags.get(0);
                Element aTag2 = aTags.get(1);
                String href = aTag1.attr("href");
                String countyCode = aTag1.text();
                String countyName = aTag2.text();

                Map<String, String> county = new HashMap<>();
                county.put("countyName", countyName);
                county.put("countyCode", countyCode);
                county.put("link",  MainApplication.url + href.split("/")[1].substring(0,2) + "/" + href);
                countyList.add(county);

                System.out.printf("(%s, '%s', '%s', NULL, NULL, '%s', NOW()),\n", countyCode, countyName, city.get(CITY_CODE), county.get("link"));
                // System.out.println(county);
            } else  {

                String countyName = tdTags.get(1).text();

                Map<String, String> county = new HashMap<>();
                county.put("countyName", countyName);
                county.put("countyCode", tdTag1);
                county.put("link", null);
                countyList.add(county);

                System.out.printf("(%s, '%s', '%s', NULL, NULL, NULL, NOW()),\n", tdTag1, countyName, city.get(CITY_CODE) );
                // System.out.println(county);
            }
        });
    }


    private static void getCityList(String s, List<Map<String, String>> cityList, Map<String, String> province) {
        String currentUrl = MainApplication.url + s;
        String htmlSourceCode = HttpClientUtil.executeGetRequest(currentUrl, headers);
        Document document = Jsoup.parse(htmlSourceCode);
        Elements citytr = document.getElementsByClass("citytr");
        // System.out.println("--------------------------------------------------");
        // System.out.println(citytr);
        citytr.forEach(element -> {
            Elements tdTags = element.getElementsByTag("td");
            // System.out.println("--------------------------------------------------");
            // System.out.println(tdTags);

            String tdTag1 = tdTags.get(0).html();
            if (tdTag1.contains("href")) {
                Elements aTags = element.getElementsByTag("a");
                Element aTag1 = aTags.get(0);
                Element aTag2 = aTags.get(1);
                String href = aTag1.attr("href");
                String cityCode = aTag1.text();
                String cityName = aTag2.text();

                Map<String, String> city = new HashMap<>();
                city.put("cityName", cityName);
                city.put("cityCode", cityCode);
                city.put("link", MainApplication.url + href);
                cityList.add(city);


                System.out.printf("(%s, '%s', '%s', NULL, NULL, '%s', NOW()),\n", cityCode, cityName, province.get(PROVINCE_CODE), city.get("link"));
                // System.out.println(city);
            } else  {

                String cityName = tdTags.get(1).text();

                Map<String, String> city = new HashMap<>();
                city.put("cityName", cityName);
                city.put("cityCode", tdTag1);
                city.put("link", null);
                cityList.add(city);
                System.out.printf("(%s, '%s', '%s', NULL, NULL, NULL, NOW()),\n", tdTag1, cityName, province.get(PROVINCE_CODE) );
                // System.out.println(city);
            }
        });
    }


    private static void getProvinceList(String url, List<Map<String, String>> provinceList) {

        // 请求得到网页源代码字符串（网页格式简单，可以直接找）
        String htmlSourceCode = HttpClientUtil.executeGetRequest(url, headers);

        // 用JSOUP解析成文档对象
        Document document = Jsoup.parse(htmlSourceCode);
        // 省级的是用这个class写的
        Elements provincetr = document.getElementsByClass("provincetr");
        provincetr.forEach(element -> {
            // System.out.println(element);

            // 下面直接找a标签就行了
            Elements aTags = element.getElementsByTag("a");
            for (Element aTag : aTags) {
                // <a href="11.html">北京市</a> 这样的结构
                String href = aTag.attr("href"); //
                String text = aTag.text();
                // System.out.println(href + " " + text);

                // 装填需要的信息
                Map<String, String> province = new HashMap<>();
                province.put(PROVINCE_NAME, text);
                province.put(PROVINCE_CODE, href.split("\\.")[0] + "0000000000"); 
                province.put("link", href);

                System.out.printf("(%s, '%s', '%s', NULL, NULL, '%s', NOW()),\n", province.get(PROVINCE_CODE), text, "0", url + href);
                provinceList.add(province);
            }
        });
    }
}
